#!/usr/bin/env python3
#-*- coding:utf-8 -*-

# 请注意，这个脚本使用的是Python3，暂时未考虑兼容性
# 抓取搜索json的结果并打印

import sys
import json
import argparse
import re
import jsonpath_rw

from urllib.request import quote,urlopen

# 默认不是debug模式
is_debug = False

def debug_log(info):
    if is_debug:
        print(info)

class DocJsValueSniffer(object):
    """
    抓取url，并从当中获取想要的数据
    """
    def __init__(self, base_url, query = None, encode = None):
        """
        base_url:抓取的url地址
        query:抓取的查询词
        """

        self.url = base_url
        # 如果url当中已经有key
        if re.search(r"&key=.+?&", base_url):
            if query:
                debug_log("warning: key in url not empty, not use your specified key")
        else:
            if not query:
                debug_log("warning: key not specified")
            else:
                self.url = base_url + "&key=" + quote(query.encode('gbk','ignore'))

        debug_log("using url:" + self.url)

    def sniff_paragraph(self, js_value_position = ""):
        """
        抓取url并解析
        """
        # 有的时候一次可能获取不到数据
        for i in range(3):
            js_data = DocJsValueSniffer.requst(self.url)
            if js_data:
                break

        if not js_data:
            print("request fail:" + self.url)
            return

        return self.parse_Paragraph(js_data, js_value_position)

    def sniff_head(self, js_value_position):
        """
        抓取url并解析
        """
        # 有的时候一次可能获取不到数据
        for i in range(3):
            js_data = DocJsValueSniffer.requst(self.url)
            if js_data:
                break

        if not js_data:
            debug_log("request fail:" + self.url)
            return

        return self.parse_head(js_data, js_value_position)

    @staticmethod
    def requst(url):
        """
        请求url
        """
        try:
            page = urlopen(url).read().decode('gbk')
        except Exception as e:
            # print('UnKnownError:',e)
            return
        try:
            json_obj = json.loads(page)
        except Exception as e:
            # print('ParseJsonError',e)
            return
        if not json_obj:
            # print('NotJsonObject')
            return

        return json_obj

    def parse_Paragraph(self, js_data, js_value_position):
        """
        解析json Paragraph数据
        """
        values = []

        if "Paragraph" not in js_data:
            return

        Paragraph = js_data['Paragraph']
        if not Paragraph:
            debug_log("Paragraph is Null:" + str(Paragraph))
            return

        jsonpath_expr = jsonpath_rw.parse(js_value_position)

        for item in Paragraph:
            if not item:
                continue

            if 'wareid' not in item:
                continue

            value = [match.value for match in jsonpath_expr.find(item)]

            wareid = item['wareid']
            values.append((wareid, value))

        return values

    def parse_head(self, js_data, js_value_position):
        """
        解析json Head数据
        """
        head = js_data['Head']
        if not head:
            return

        jsonpath_expr = jsonpath_rw.parse(js_value_position)
        value = [match.value for match in jsonpath_expr.find(head)]

        return value

    @staticmethod
    def parse_js_data_with_position(js_data, position):
        """
        position like:
             Sort.debugMsg.SearcherModelDebug.HotModelPop.pop_hot_score
             Sort.MDBG.HotModelPop[0].Weight
        """
        position_list = position.split(".")

        value = js_data

        for pos in position_list:
            pos = pos.split("[")

            if len(pos) == 2:
                # 以括号包裹的表示数组类型
                key = pos[0]
                index = int(pos[1].strip("]"))

                if key not in value:
                    return

                value = value[key]
                # 越界
                if len(value) <= index:
                    return

                value = value[index]
            else:
                key = pos[0]
                if key not in value:
                    return
                value = value[key]

            if not value:
                return

        return value

def main():
    parser = argparse.ArgumentParser(description =
        """这个脚本用以抓取线上的json数据，
        抓取的数据按照json结果的顺序排列。注意：url当中添加enc_url_gbk=yes
        可以避免title当中包含特殊字符解析错误""")

    parser.add_argument("-a","--head",
            help = "爬取头部信息，参数为头部信息的位置，如query.HcCid1s")
    parser.add_argument("-p","--paragraph",
            help = "爬取商品信息，参数为商品信息的位置，如brand_id")
    parser.add_argument("-k","--key",
            help = "要请求的key，默认以utf8编码，若是gbk编码，请使用--encode")
    parser.add_argument("-e","--encode",
            help = "参数key的编码，如gbk，utf8等")
    parser.add_argument("-u","--url",
            help = "要请求的url，若url当中不包含key，请使用--key指定")
    parser.add_argument("-d","--debug",
            help = "是否输出debug信息", action = "store_true")

    args = parser.parse_args()

    if not args.url:
        print("未指定url，无法爬取")
        return

    global is_debug
    is_debug = args.debug

    doc_js_sniffer = DocJsValueSniffer(args.url, query = args.key, encode = args.encode)

    if args.head:
        data = doc_js_sniffer.sniff_head(args.head)
    elif args.paragraph:
        data = doc_js_sniffer.sniff_paragraph(args.paragraph)

    if data:
        print(repr(data))

    return

if __name__=='__main__':
    main()


